package at.jaki.ubb.crawling.demo.dataTransformer;

import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;

import at.jaki.ubb.crawling.api.DataTransformer;
import at.jaki.ubb.crawling.demo.model.StringEntity;
import at.jaki.ubb.crawling.model.DataTransformerResult;
import at.jaki.ubb.crawling.model.RawData;
import at.jaki.ubb.crawling.stlib.utils.WebPageMinerUtils;

public class RealitateaLinkExtractor implements DataTransformer<String, String, StringEntity, String> {

	private static final Logger LOG = Logger.getLogger(RealitateaLinkExtractor.class);

	private static final String SEARCH_RESULT_PATTERN = "href=[\"](.*?)[\"]";
	private static final int MIN_LINK_LENGTH = 5;
	private static final String REALITATEA = "http://realitatea.ro";

	public DataTransformerResult<String, String, StringEntity> transformData(RawData<String> rawData) {

		String hitsSection = WebPageMinerUtils.getRegion("<div class=\"sectionPage\">", rawData.getData());

		Pattern pattern = Pattern.compile(SEARCH_RESULT_PATTERN);
		Matcher m = pattern.matcher(hitsSection);

		Collection<StringEntity> coll = new HashSet<StringEntity>();

		while (m.find()) {
			String link = m.group(1);
			if (link.length() > MIN_LINK_LENGTH) {
				String l = link.startsWith("http") ? link : REALITATEA + link;
				if (!l.contains("/cauta?") && !l.contains("javascript")) {
					if (l.matches(".*[0-9]([.]html)?")) {
						coll.add(new StringEntity(l));
					}
				}
			}
		}

		LOG.info("Extracted links: " + coll);

		DataTransformerResult<String, String, StringEntity> dtr = new DataTransformerResult<String, String, StringEntity>(null, coll, null);
		return dtr;
	}

	public void init() {
		// nothing to initialize
	}

	public List<String> getTypes() {
		return Collections.singletonList(PageClassifier.TYPE_REALITATEA_SEARCH_PAGE);
	}

}
